{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1d_10 (Conv1D)           (None, 28, 64)            6464      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_4 (MaxPooling1 (None, 14, 64)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_11 (Conv1D)           (None, 10, 128)           41088     \n",
      "_________________________________________________________________\n",
      "conv1d_12 (Conv1D)           (None, 6, 64)             41024     \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d_4 ( (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 2)                 130       \n",
      "=================================================================\n",
      "Total params: 88,706\n",
      "Trainable params: 88,706\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 1600 samples, validate on 400 samples\n",
      "Epoch 1/5\n",
      "1600/1600 [==============================] - 9s 6ms/step - loss: 6.0490 - acc: 0.5831 - val_loss: 3.9472 - val_acc: 0.6850\n",
      "Epoch 2/5\n",
      "1600/1600 [==============================] - 4s 2ms/step - loss: 2.6232 - acc: 0.8038 - val_loss: 1.5293 - val_acc: 0.8800\n",
      "Epoch 3/5\n",
      "1600/1600 [==============================] - 4s 2ms/step - loss: 1.2782 - acc: 0.8837 - val_loss: 0.7069 - val_acc: 0.8900\n",
      "Epoch 4/5\n",
      "1600/1600 [==============================] - 4s 3ms/step - loss: 0.2597 - acc: 0.9425 - val_loss: 0.2699 - val_acc: 0.9325\n",
      "Epoch 5/5\n",
      "1600/1600 [==============================] - 4s 2ms/step - loss: 0.1408 - acc: 0.9500 - val_loss: 0.1697 - val_acc: 0.9375\n",
      "Test loss: 0.16973233073949814\n",
      "Test accuracy: 0.9375\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pickle\n",
    "import keras\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import shuffle\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.callbacks import TensorBoard\n",
    "from keras.utils import print_summary\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense,Flatten, Conv2D,Conv1D , Convolution1D , Activation\n",
    "from keras.layers import MaxPooling2D, Dropout,MaxPooling1D,GlobalAveragePooling1D\n",
    "\n",
    "from data_utils import loadFromPickle\n",
    "\n",
    "from keras.utils import np_utils\n",
    "\n",
    "def prepress_labels(labels):\n",
    "    labels = np_utils.to_categorical(labels) # one-hot编码 把类别id转换为表示当前类别的向量，比如0 1 2 =》 [[1 0 0] [0 1 0] [0 0 1]]\n",
    "    return labels\n",
    "\n",
    "def keras_model_1dconv(input_shape,num_classes):\n",
    "    model = Sequential()\n",
    "    model.add(Conv1D(64, 5, activation='relu', input_shape=input_shape))\n",
    "    # model.add(Conv1D(256, 10, activation='relu'))\n",
    "    model.add(MaxPooling1D(2))\n",
    "    model.add(Conv1D(128, 5, activation='relu'))\n",
    "    model.add(Conv1D(64, 5, activation='relu'))\n",
    "    model.add(GlobalAveragePooling1D())\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(num_classes, activation='softmax'))\n",
    "    print(model.summary())\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy'])\n",
    "    filepath = \"1dconv.h5\"\n",
    "    checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "    callbacks_list = [checkpoint]\n",
    "    return model, callbacks_list\n",
    "\n",
    "    # print(model.summary())   \n",
    "\n",
    "def keras_model2(input_shape,num_classes):\n",
    "    # 构建模型\n",
    "    model = Sequential()\n",
    "    model.add(Dense(512, activation='relu',input_shape=input_shape))\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "    model.add(Dense(num_classes, activation='softmax'))\n",
    "    # [编译模型] 配置模型，损失函数采用交叉熵，优化采用Adadelta，将识别准确率作为模型评估\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy'])\n",
    "    filepath = \"model1.h5\"\n",
    "    checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "    callbacks_list = [checkpoint]\n",
    "    return model, callbacks_list\n",
    "\n",
    "def keras_model1(input_shape,num_of_classes):\n",
    "    # 构建模型\n",
    "    model = Sequential()\n",
    "    model.add(Dense(128, activation='relu',input_shape=input_shape))\n",
    "    model.add(keras.layers.core.Reshape((128,1)))\n",
    "    model.add(Conv1D(64, 20, activation='relu'))\n",
    "    # model.add(MaxPooling1D(2))\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "    model.add(GlobalAveragePooling1D())\n",
    "    model.add(Dense(num_of_classes, activation='softmax'))\n",
    "    # [编译模型] 配置模型，损失函数采用交叉熵，优化采用Adadelta，将识别准确率作为模型评估\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy'])\n",
    "    filepath = \"asr_mfcc_dense_model.h5\"\n",
    "    checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "    callbacks_list = [checkpoint]\n",
    "    return model, callbacks_list\n",
    "\n",
    "def keras_model(input_shape,num_of_classes):\n",
    "    num_of_classes = num_of_classes\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(32, (5, 5), input_shape=input_shape, activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))\n",
    "    model.add(Conv2D(64, (5, 5), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))\n",
    "\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(140, activation='relu'))\n",
    "    model.add(Dropout(0.6))\n",
    "    model.add(Dense(70, activation='relu'))\n",
    "    model.add(Dropout(0.6))\n",
    "    model.add(Dense(num_of_classes, activation='softmax'))\n",
    "\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    filepath = \"asr_mfcc_conv2d_model.h5\"\n",
    "    checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "    callbacks_list = [checkpoint]\n",
    "\n",
    "    return model, callbacks_list    \n",
    "\n",
    "def test_model():\n",
    "    features, labels = loadFromPickle()\n",
    "    features, labels = shuffle(features, labels)\n",
    "    features=features.reshape(features.shape[0],64,40,1)\n",
    "    labels=prepress_labels(labels)\n",
    "    train_x, test_x, train_y, test_y = train_test_split(features, labels, random_state=0,\n",
    "                                                        test_size=0.1)\n",
    "    model, callbacks_list = keras_model((64,40,1,),len(labels[0]))\n",
    "    print_summary(model)\n",
    "    model.fit(train_x, train_y, batch_size=128, epochs=5, verbose=1, validation_data=(test_x, test_y))\n",
    "\n",
    "    # 开始评估模型效果 # verbose=0为不输出日志信息\n",
    "    score = model.evaluate(test_x, test_y, verbose=0)\n",
    "    print('Test loss:', score[0])\n",
    "    print('Test accuracy:', score[1]) # 准确度\n",
    "\n",
    "    model.save('asr_mfcc_conv2d_model.h5') # 保存训练模型\n",
    "\n",
    "def test_model1(input_length):\n",
    "    features, labels = loadFromPickle()\n",
    "    features, labels = shuffle(features, labels)\n",
    "    features=features.reshape(features.shape[0],input_length)\n",
    "    labels=prepress_labels(labels)\n",
    "    train_x, test_x, train_y, test_y = train_test_split(features, labels, random_state=0,\n",
    "                                                        test_size=0.2)\n",
    "    model, callbacks_list = keras_model1((input_length,),len(labels[0]))\n",
    "    print_summary(model)\n",
    "    model.fit(train_x, train_y, batch_size=128, epochs=20, verbose=1, validation_data=(test_x, test_y),\n",
    "    \tcallbacks=[TensorBoard(log_dir=\"TensorBoard\")])\n",
    "\n",
    "    score = model.evaluate(test_x, test_y, verbose=0)\n",
    "    print('Test loss:', score[0])\n",
    "    print('Test accuracy:', score[1]) # 准确度\n",
    "    model.save('test_model1.h5') # 保存训练模型\n",
    "\n",
    "def test_model2(input_length):\n",
    "    features, labels = loadFromPickle()\n",
    "    features, labels = shuffle(features, labels)\n",
    "    features=features.reshape(features.shape[0],input_length)\n",
    "    labels=prepress_labels(labels)\n",
    "    train_x, test_x, train_y, test_y = train_test_split(features, labels, random_state=0,\n",
    "                                                        test_size=0.2)\n",
    "    model, callbacks_list = keras_model2((input_length,),len(labels[0]))\n",
    "    print_summary(model)\n",
    "    model.fit(train_x, train_y, batch_size=128, epochs=20, verbose=1, validation_data=(test_x, test_y))\n",
    "\n",
    "    score = model.evaluate(test_x, test_y, verbose=0)\n",
    "    print('Test loss:', score[0])\n",
    "    print('Test accuracy:', score[1]) # 准确度\n",
    "    model.save('test_model2.h5') # 保存训练模型\n",
    "\n",
    "def test_1dconv(d_x,d_y):\n",
    "    features, labels = loadFromPickle()\n",
    "    features, labels = shuffle(features, labels)\n",
    "    # features=features.reshape(features.shape[0],64,40)\n",
    "    features=features.reshape(features.shape[0],d_x,d_y)\n",
    "    labels=prepress_labels(labels)\n",
    "    train_x, test_x, train_y, test_y = train_test_split(features, labels, random_state=0,\n",
    "                                                        test_size=0.2)\n",
    "    model, callbacks_list = keras_model_1dconv((d_x,d_y),len(labels[0]))\n",
    "    #print_summary(model)\n",
    "    model.fit(train_x, train_y, batch_size=128, epochs=5, verbose=1, validation_data=(test_x, test_y))\n",
    "\n",
    "    score = model.evaluate(test_x, test_y, verbose=0)\n",
    "    print('Test loss:', score[0])\n",
    "    print('Test accuracy:', score[1]) # 准确度\n",
    "    model.save('test_1dconv.h5') # 保存训练模型\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    test_1dconv(32,20)\n",
    "    # test_model2()\n",
    "    # test_model1()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Listening...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:snowboy:Keyword 1 detected at time: 2019-05-13 14:19:49\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Listening...\n",
      "may be  yes , probab:  0.6989424\n",
      "小白：你说的是yes\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:snowboy:Keyword 1 detected at time: 2019-05-13 14:20:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Listening...\n",
      "may be  no , probab:  1.0\n",
      "小白：你说的是no\n",
      "stop\n"
     ]
    }
   ],
   "source": [
    "#!coding:utf-8\n",
    "import sys\n",
    "import os\n",
    "from keras.models import load_model\n",
    "from data_utils import load_label_name,get_mfcc,get_pcm\n",
    "import numpy as np\n",
    "\n",
    "root_dir = \"/xiaobai/\"\n",
    "sys.path.append(root_dir)\n",
    "from xiaobai import XiaoBai,BaseSkill\n",
    "\n",
    "# model = load_model('asr_model.h5') # 加载训练模型\n",
    "# model = load_model('asr_mfcc_conv1d_model.h5') # 加载训练模型\n",
    "model = load_model('test_1dconv.h5') # 加载训练模型\n",
    "# model = load_model('asr_mfcc_dense_model.h5') # 加载训练模型\n",
    "label_names = load_label_name()\n",
    "speak = None\n",
    "def predict(model):\n",
    "    global speak\n",
    "    X=get_mfcc('record.wav',samples=16000)\n",
    "    # X=get_pcm('record.wav',samples=32000)\n",
    "    #print(X.shape)\n",
    "    #test dense model\n",
    "    # X = np.reshape(np.array(X),(-1,32000))\n",
    "\n",
    "    # test conv1d model\n",
    "    X = np.reshape(np.array(X),(-1,32,20))\n",
    "    # test conv2d model\n",
    "    # X = np.reshape(np.array(X),(-1,64,40))    \n",
    "    pred_probab = model.predict(X)\n",
    "    pred_class = list(pred_probab[0]).index(max(pred_probab[0]))\n",
    "    print(\"may be \" , label_names[pred_class] , \", probab: \" ,  pred_probab[0][pred_class])\n",
    "    if speak is not None:\n",
    "        speak(\"你说的是\"+label_names[pred_class])\n",
    "    formated = list( map(lambda x,i : (x.item(),label_names[i]) , pred_probab[0],[i for i in range(len(label_names))]) )\n",
    "    lists = sorted(formated,reverse=True)\n",
    "    top5 = lists[:5]\n",
    "    #print(\"top5 :\",top5)\n",
    "    return lists\n",
    "def callback():\n",
    "    print(\"Listening...\")\n",
    "    os.system(\"arecord -d %d -r 16000 -c 1 -t wav -f S16_LE record.wav\" % (1,) )   \n",
    "    predict(model)\n",
    "def main():\n",
    "    global speak\n",
    "    keyword_model = root_dir+'resources/小白.pmdl'\n",
    "    xiaobai = XiaoBai(keyword_model=keyword_model,callback=callback)\n",
    "    speak = xiaobai.speak\n",
    "    xiaobai.listen_for_keyword()\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
